{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.2 Python 使用入门"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2.1 基本命令"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:10:46.015433Z",
     "start_time": "2018-12-26T16:10:46.003428Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!\n"
     ]
    }
   ],
   "source": [
    "print (\"Hello World!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:10:46.666754Z",
     "start_time": "2018-12-26T16:10:46.663742Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!\n"
     ]
    }
   ],
   "source": [
    "x = 1\n",
    "if x == 1:\n",
    "    print (\"Hello World!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:10:47.153358Z",
     "start_time": "2018-12-26T16:10:47.150852Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!\n"
     ]
    }
   ],
   "source": [
    "# 在前面加上#，代表注释\n",
    "print (\"Hello World!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2.2 数据类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:10:54.016816Z",
     "start_time": "2018-12-26T16:10:54.013782Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python Web Scrappy by Santos\n"
     ]
    }
   ],
   "source": [
    "string1 = 'Python Web Scrappy'\n",
    "string2 = \"by Santos\"\n",
    "string3 = string1 + \" \" + string2\n",
    "print (string3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:10:58.779532Z",
     "start_time": "2018-12-26T16:10:58.776502Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm Santos. I love \"python\".\n"
     ]
    }
   ],
   "source": [
    "string = \"I\\'m Santos. I love \\\"python\\\".\"\n",
    "print (string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "int1 = 7\n",
    "float1 = 7.5\n",
    "trans_int = int(float1)\n",
    "print (trans_int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:11:38.330654Z",
     "start_time": "2018-12-26T16:11:38.327646Z"
    }
   },
   "outputs": [],
   "source": [
    "list1 = ['Python', 'Web', 'Scrappy']\n",
    "list2 = [1, 2, 3, 4, 5]\n",
    "list3 = [\"a\", 2, \"c\", 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:11:39.318075Z",
     "start_time": "2018-12-26T16:11:39.315088Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list1[0]:  Python\n",
      "list2[1:3]:  [2, 3]\n"
     ]
    }
   ],
   "source": [
    "print (\"list1[0]: \" , list1[0])\n",
    "print (\"list2[1:3]: \" , list2[1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:25:38.520971Z",
     "start_time": "2018-12-26T16:25:38.517964Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Python', 'new', 'Scrappy']\n"
     ]
    }
   ],
   "source": [
    "list1[1] = \"new\"\n",
    "print (list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:25:39.262945Z",
     "start_time": "2018-12-26T16:25:39.259937Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Python', 'new', 'Scrappy', 'by Santos']\n"
     ]
    }
   ],
   "source": [
    "list1.append(\"by Santos\")\n",
    "print (list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:25:56.384476Z",
     "start_time": "2018-12-26T16:25:56.381467Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alex\n",
      "{'Name': 'Alex', 'Age': 7, 'Class': 'First'}\n"
     ]
    }
   ],
   "source": [
    "namebook = {\"Name\": \"Alex\", \"Age\": 7, \"Class\": \"First\"}\n",
    "print (namebook[\"Name\"]) #可以把相应的键值放入方括号，提取值\n",
    "print (namebook) #也可以直接输出整个字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:01.238384Z",
     "start_time": "2018-12-26T16:26:01.236403Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name Alex\n",
      "Age 7\n",
      "Class First\n"
     ]
    }
   ],
   "source": [
    "#循环提取整个dictionary的key和value\n",
    "for key, value in namebook.items():\n",
    "    print (key, value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:03.289840Z",
     "start_time": "2018-12-26T16:26:03.286831Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Name': 'Tom', 'Age': 7, 'Class': 'First', 'Gender': 'M'}\n"
     ]
    }
   ],
   "source": [
    "# 直接修改值和添加一个键值\n",
    "namebook[\"Name\"] = \"Tom\"\n",
    "namebook [\"Gender\"] = \"M\"\n",
    "print (namebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2.3 条件语句和循环语句\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:21.040042Z",
     "start_time": "2018-12-26T16:26:21.037034Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are studying python.\n"
     ]
    }
   ],
   "source": [
    "book = \"python\"  #定义字符串book\n",
    "if book == \"python\":   #判断变量是否为'python'\n",
    "    print (\"You are studying python.\")  #条件成立时输出\n",
    "else:\n",
    "    print (\"Wrong.\") #条件不成立时输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:21.503274Z",
     "start_time": "2018-12-26T16:26:21.501268Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are studying java.\n"
     ]
    }
   ],
   "source": [
    "book = \"java\"  #定义字符串book\n",
    "if book == \"python\":   #判断变量是否为'python'\n",
    "    print (\"You are studying python.\")  #条件成立时输出\n",
    "elif book == \"java\":  #判断变量是否为'java '\n",
    "    print (\"You are studying java.\") #条件成立时输出\n",
    "else:\n",
    "    print (\"Wrong.\") #条件不成立时输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:28.776625Z",
     "start_time": "2018-12-26T16:26:28.773619Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beijing\n",
      "Shanghai\n",
      "Guangzhou\n"
     ]
    }
   ],
   "source": [
    "citylist = [\"Beijing\", \"Shanghai\", \"Guangzhou\"]\n",
    "for eachcity in citylist:\n",
    "    print (eachcity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:41.654873Z",
     "start_time": "2018-12-26T16:26:41.652868Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beijing\n",
      "Shanghai\n",
      "Guangzhou\n"
     ]
    }
   ],
   "source": [
    "citylist = [\"Beijing\", \"Shanghai\", \"Guangzhou\"]\n",
    "for i in range(len(citylist)):\n",
    "    print (citylist[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:26:42.496109Z",
     "start_time": "2018-12-26T16:26:42.493101Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "while count < 3:\n",
    "    print (count) #打印出 0,1,2\n",
    "    count += 1 #与 count = count + 1 一样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2.4 函数(Functions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "print (sum([1,2,3,4])) #对系列进行求和计算\n",
    "print (abs(-1)) # 返回数字的绝对值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "#定义函数\n",
    "def calulus (x):\n",
    "    y = x + 1\n",
    "    return y\n",
    "\n",
    "#调用函数\n",
    "result = calulus(2)\n",
    "print (result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apple banana orange\n"
     ]
    }
   ],
   "source": [
    "#定义函数\n",
    "def fruit_function (fruit1, fruit2):\n",
    "    fruits = fruit1 + \" \" + fruit2[0] + \" \" + fruit2[1]\n",
    "    return fruits\n",
    "\n",
    "#调用函数\n",
    "result = fruit_function(\"apple\", [\"banana\", \"orange\"])\n",
    "print (result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2.5  面向对象编程\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-11T09:31:51.290201Z",
     "start_time": "2017-07-11T09:31:51.274622Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "santos\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "class Person:   # 创建类\n",
    "    def __init__(self, name, age): #__init__()方法称为类的构造方法\n",
    "        self.name = name\n",
    "        self.age = age\n",
    " \n",
    "    def detail(self): #通过self调用被封装的内容\n",
    "        print (self.name)\n",
    "        print (self.age)\n",
    "\n",
    "obj1 = Person('santos', 18)\n",
    "obj1.detail()  # Python将obj1传给self参数，即：obj1.detail(obj1)，此时内部self＝obj1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "santos\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "def detail(name,age): \n",
    "    print (name)\n",
    "    print (age)\n",
    "obj1 = detail('santos', 18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-11T09:32:29.618086Z",
     "start_time": "2017-07-11T09:32:29.612355Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Person:   # 创建类\n",
    "    def __init__(self, name, age): #__init__()方法称为类的构造方法\n",
    "        self.name = name\n",
    "        self.age = age\n",
    "        \n",
    "obj1 = Person('santos', 18) #将\"santos\"和 18 分别封装到 obj1 及 self的 name和age属性"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 封装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "santos\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "class Person:   # 创建类\n",
    "    def __init__(self, name, age): #__init__()方法称为类的构造方法\n",
    "        self.name = name\n",
    "        self.age = age\n",
    "        \n",
    "obj1 = Person('santos', 18) #将\"santos\"和 18 分别封装到 obj1 及 self的 name和age属性\n",
    "print (obj1.name)    # 直接调用obj1对象的name属性\n",
    "print (obj1.age)     # 直接调用obj1对象的age属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-11T09:34:20.495707Z",
     "start_time": "2017-07-11T09:34:20.485891Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "santos\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "class Person:   # 创建类\n",
    "    def __init__(self, name, age): #__init__()方法称为类的构造方法\n",
    "        self.name = name\n",
    "        self.age = age\n",
    " \n",
    "    def detail(self): #通过self调用被封装的内容\n",
    "        print (self.name)\n",
    "        print (self.age)\n",
    "\n",
    "obj1 = Person('santos', 18)\n",
    "obj1.detail()  # Python将obj1传给self参数，即：obj1.detail(obj1)，此时内部self＝obj1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 继承"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-11T09:34:44.252777Z",
     "start_time": "2017-07-11T09:34:44.243993Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class 猫： \n",
    "    def 喵喵叫(self):\n",
    "        print ('喵喵叫')\n",
    "    def 吃(self):\n",
    "        # do something\n",
    "    def 喝(self):\n",
    "        # do something \n",
    "    def 拉(self):\n",
    "        # do something \n",
    "    def 撒(self):\n",
    "        # do something\n",
    "\n",
    "class 狗：\n",
    "    def 汪汪叫(self):\n",
    "        print ('汪汪叫')\n",
    "    def 吃(self):\n",
    "        # do something\n",
    "    def 喝(self):\n",
    "        # do something \n",
    "    def 拉(self):\n",
    "        # do something \n",
    "    def 撒(self):\n",
    "        # do something"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-26T16:30:34.067952Z",
     "start_time": "2018-12-26T16:30:34.062937Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "小白家的小黑猫 吃 \n",
      "喵喵叫\n",
      "胖子家的小瘦狗 吃 \n",
      "汪汪叫\n"
     ]
    }
   ],
   "source": [
    "class Animal:\n",
    "    def eat(self):\n",
    "        print (\"%s 吃 \" %self.name)\n",
    "    def drink(self):\n",
    "        print (\"%s 喝 \" %self.name) \n",
    "    def shit(self):\n",
    "        print (\"%s 拉 \" %self.name) \n",
    "    def pee(self):\n",
    "        print (\"%s 撒 \" %self.name)\n",
    "class Cat(Animal): \n",
    "    def __init__(self, name):\n",
    "        self.name = name \n",
    "    def cry(self):\n",
    "        print ('喵喵叫')\n",
    "class Dog(Animal): \n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "    def cry(self):\n",
    "        print ('汪汪叫')\n",
    "        \n",
    "c1 = Cat('小白家的小黑猫')\n",
    "c1.eat()\n",
    "c1.cry()\n",
    " \n",
    "d1 = Dog('胖子家的小瘦狗')\n",
    "d1.eat()\n",
    "d1.cry()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "division by zero\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    result = 5/0 #除以0会产生运算错误\n",
    "except Exception as e: #出现错误会执行except\n",
    "    print (e)   #把错误打印出来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    result = 5/0 #除以0会产生运算错误\n",
    "except: #出现错误会执行except\n",
    "    pass    #空语句，不做任何事情"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 2.3 编写你的第一个简单爬虫"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.1 第一步：获取页面\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "<!DOCTYPE html>\n",
      "<html lang=\"zh-CN\">\n",
      "<head>\n",
      "<meta charset=\"UTF-8\">\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n",
      "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1, maximum-scale=1\">\n",
      "<title>唐松 Santos</title>\n",
      "<meta name=\"description\" content=\"这是唐松 Santos的个人博客，《Python 网络爬虫: 从入门到实践》作者\" />\n",
      "<meta name=\"keywords\" content=\"唐松 Santos, Python, 网络爬虫, Python 网络爬虫: 从入门到实践, Python 爬虫, 大数据\" />\n",
      "<link rel=\"apple-touch-icon\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/images/icon_32.png\">\n",
      "<link rel=\"apple-touch-icon\" sizes=\"152x152\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/images/icon_152.png\">\n",
      "<link rel=\"apple-touch-icon\" sizes=\"167x167\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/images/icon_167.png\">\n",
      "<link rel=\"apple-touch-icon\" sizes=\"180x180\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/images/icon_180.png\">\n",
      "<link rel=\"icon\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/images/icon_32.png\" type=\"image/x-icon\">\n",
      "<link rel=\"stylesheet\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/css/bootstrap.min.css\">\n",
      "<link rel=\"stylesheet\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/css/font-awesome.min.css\">\n",
      "<script type=\"text/javascript\" src=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/js/jquery.min.js\"></script>\n",
      "<script type=\"text/javascript\" src=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/js/bootstrap.min.js\"></script>\n",
      "<link rel=\"stylesheet\" href=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/style.css\">\n",
      "<link rel=\"pingback\" href=\"http://www.santostang.com/xmlrpc.php\" />\n",
      "<link rel=\"icon\" href=\"http://www.santostang.com/wp-content/uploads/2018/07/cropped-IMG_0423-min-32x32.jpg\" sizes=\"32x32\" />\n",
      "<link rel=\"icon\" href=\"http://www.santostang.com/wp-content/uploads/2018/07/cropped-IMG_0423-min-192x192.jpg\" sizes=\"192x192\" />\n",
      "<link rel=\"apple-touch-icon-precomposed\" href=\"http://www.santostang.com/wp-content/uploads/2018/07/cropped-IMG_0423-min-180x180.jpg\" />\n",
      "<meta name=\"msapplication-TileImage\" content=\"http://www.santostang.com/wp-content/uploads/2018/07/cropped-IMG_0423-min-270x270.jpg\" />\n",
      "<style type=\"text/css\">\n",
      "a{color:#1e73be}\n",
      "a:hover{color:#2980b9!important}\n",
      "#header{background-color:#1e73be}\n",
      ".widget .widget-title::after{background-color:#1e73be}\n",
      ".uptop{border-left-color:#1e73be}\n",
      "#titleBar .toggle:before{background:#1e73be}\n",
      "</style>\n",
      "</head>\n",
      "\n",
      "<body>\n",
      "<header id=\"header\">\n",
      "    <div class=\"avatar\"><a href=\"http://www.santostang.com\" title=\"唐松Santos\"><img src=\"http://www.santostang.com/wp-content/uploads/2018/07/IMG_0423-min.jpg\" alt=\"唐松Santos\" class=\"img-circle\" width=\"50%\"></a></div>\n",
      "    <h1 id=\"name\">唐松Santos</h1>\n",
      "    <div class=\"sns\">\n",
      "                <a href=\"https://www.zhihu.com/people/santostang\" target=\"_blank\" rel=\"nofollow\" title=\"Weibo\"><i class=\"fa fa-weibo\" aria-hidden=\"true\"></i></a>        <a href=\"https://www.linkedin.com/in/santostang\" target=\"_blank\" rel=\"nofollow\" title=\"twitter\"><i class=\"fa fa-linkedin\" aria-hidden=\"true\"></i></a>                <a href=\"https://github.com/Santostang\" target=\"_blank\" rel=\"nofollow\" title=\"GitHub\"><i class=\"fa fa-github\" aria-hidden=\"true\"></i></a>    </div>\n",
      "    <div class=\"nav\">\n",
      "        <ul><li><a href=\"http://www.santostang.com/\">首页</a></li>\n",
      "<li><a href=\"http://www.santostang.com/sample-page/\">关于我</a></li>\n",
      "<li><a href=\"http://www.santostang.com/python%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%e4%bb%a3%e7%a0%81/\">爬虫书代码</a></li>\n",
      "<li><a href=\"http://www.santostang.com/%e5%8a%a0%e6%88%91%e5%be%ae%e4%bf%a1/\">加我微信</a></li>\n",
      "<li><a href=\"https://santostang.github.io/\">EnglishSite</a></li>\n",
      "</ul>    </div>\n",
      "        <div class=\"weixin\">\n",
      "        <img src=\"http://www.santostang.com/wp-content/uploads/2018/07/qrcode_for_gh_370f70791e19_344-1.jpg\" alt=\"微信公众号\" width=\"50%\">\n",
      "        <p>微信公众号</p>\n",
      "    </div>\n",
      "    </header>\n",
      "\n",
      "<div id=\"main\">\n",
      "    <div class=\"row box\">\n",
      "        <div class=\"col-md-8\">\n",
      "                                <article class=\"article-list-1 clearfix\">\n",
      "                <header class=\"clearfix\">\n",
      "                    <h1 class=\"post-title\"><a href=\"http://www.santostang.com/2018/07/15/4-3-%e9%80%9a%e8%bf%87selenium-%e6%a8%a1%e6%8b%9f%e6%b5%8f%e8%a7%88%e5%99%a8%e6%8a%93%e5%8f%96/\">4.3 通过selenium 模拟浏览器抓取</a></h1>\n",
      "                    <div class=\"post-meta\">\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-calendar\"></i> 07月15日</span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-folder-open-o\"></i> <a href=\"http://www.santostang.com/category/uncategorized/\" rel=\"category tag\">未分类</a></span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-commenting-o\"></i> <a href=\"http://www.santostang.com/2018/07/15/4-3-%e9%80%9a%e8%bf%87selenium-%e6%a8%a1%e6%8b%9f%e6%b5%8f%e8%a7%88%e5%99%a8%e6%8a%93%e5%8f%96/#respond\">没有评论</a></span>\n",
      "                        <span class=\"meta-span hidden-xs\"><i class=\"fa fa-tags\" aria-hidden=\"true\"></i> </span>\n",
      "                    </div>\n",
      "                </header>\n",
      "                <div class=\"post-content clearfix\">\n",
      "                    <p>4.3 通过selenium 模拟浏览器抓取\n",
      "\n",
      "在上述的例子中，使用Chrome“检查”功能找到源地址还十分容易。但是有一些网站非常复杂，例如前面的天猫产品评论，使用“检查”功能很难找到调用的网页地址。除此之外，有一些数据...</p>\n",
      "                </div>\n",
      "            </article>\n",
      "                                                <article class=\"article-list-1 clearfix\">\n",
      "                <header class=\"clearfix\">\n",
      "                    <h1 class=\"post-title\"><a href=\"http://www.santostang.com/2018/07/14/4-2-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80%e6%8a%93%e5%8f%96/\">4.2 解析真实地址抓取</a></h1>\n",
      "                    <div class=\"post-meta\">\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-calendar\"></i> 07月14日</span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-folder-open-o\"></i> <a href=\"http://www.santostang.com/category/python-%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"category tag\">Python 网络爬虫</a></span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-commenting-o\"></i> <a href=\"http://www.santostang.com/2018/07/14/4-2-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80%e6%8a%93%e5%8f%96/#respond\">没有评论</a></span>\n",
      "                        <span class=\"meta-span hidden-xs\"><i class=\"fa fa-tags\" aria-hidden=\"true\"></i> <a href=\"http://www.santostang.com/tag/ajax/\" rel=\"tag\">ajax</a>,<a href=\"http://www.santostang.com/tag/python/\" rel=\"tag\">python</a>,<a href=\"http://www.santostang.com/tag/%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"tag\">网络爬虫</a>,<a href=\"http://www.santostang.com/tag/%e7%bd%91%e9%a1%b5%e7%88%ac%e8%99%ab/\" rel=\"tag\">网页爬虫</a>,<a href=\"http://www.santostang.com/tag/%e8%a7%a3%e6%9e%90%e5%9c%b0%e5%9d%80/\" rel=\"tag\">解析地址</a></span>\n",
      "                    </div>\n",
      "                </header>\n",
      "                <div class=\"post-content clearfix\">\n",
      "                    <p>由于网易云跟帖停止服务，现在已经在此处中更新了新写的第四章。请参照文章：\n",
      "\n",
      "4.2 解析真实地址抓取\n",
      "\n",
      "虽然数据并没有出现在网页源代码中，我们也可以找到数据的真实地址，请求这个真实地址也可以获得想要的数据...</p>\n",
      "                </div>\n",
      "            </article>\n",
      "                                                <article class=\"article-list-1 clearfix\">\n",
      "                <header class=\"clearfix\">\n",
      "                    <h1 class=\"post-title\"><a href=\"http://www.santostang.com/2018/07/14/%e7%ac%ac%e5%9b%9b%e7%ab%a0%ef%bc%9a%e5%8a%a8%e6%80%81%e7%bd%91%e9%a1%b5%e6%8a%93%e5%8f%96-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80-selenium/\">第四章- 动态网页抓取 (解析真实地址 + selenium)</a></h1>\n",
      "                    <div class=\"post-meta\">\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-calendar\"></i> 07月14日</span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-folder-open-o\"></i> <a href=\"http://www.santostang.com/category/python-%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"category tag\">Python 网络爬虫</a></span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-commenting-o\"></i> <a href=\"http://www.santostang.com/2018/07/14/%e7%ac%ac%e5%9b%9b%e7%ab%a0%ef%bc%9a%e5%8a%a8%e6%80%81%e7%bd%91%e9%a1%b5%e6%8a%93%e5%8f%96-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80-selenium/#respond\">没有评论</a></span>\n",
      "                        <span class=\"meta-span hidden-xs\"><i class=\"fa fa-tags\" aria-hidden=\"true\"></i> <a href=\"http://www.santostang.com/tag/ajax/\" rel=\"tag\">ajax</a>,<a href=\"http://www.santostang.com/tag/javascript/\" rel=\"tag\">javascript</a>,<a href=\"http://www.santostang.com/tag/python/\" rel=\"tag\">python</a>,<a href=\"http://www.santostang.com/tag/sselenium/\" rel=\"tag\">sselenium</a>,<a href=\"http://www.santostang.com/tag/%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"tag\">网络爬虫</a></span>\n",
      "                    </div>\n",
      "                </header>\n",
      "                <div class=\"post-content clearfix\">\n",
      "                    <p>由于网易云跟帖停止服务，现在已经在此处中更新了新写的第四章。请参照文章：\n",
      "\n",
      "前面爬取的网页均为静态网页，这样的网页在浏览器中展示的内容都在 HTML 源代码中。但是，由于主流网站都使用JavaScript 展现网页内...</p>\n",
      "                </div>\n",
      "            </article>\n",
      "                                                <article class=\"article-list-1 clearfix\">\n",
      "                <header class=\"clearfix\">\n",
      "                    <h1 class=\"post-title\"><a href=\"http://www.santostang.com/2018/07/11/%e3%80%8a%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%ef%bc%9a%e4%bb%8e%e5%85%a5%e9%97%a8%e5%88%b0%e5%ae%9e%e8%b7%b5%e3%80%8b%e4%b8%80%e4%b9%a6%e5%8b%98%e8%af%af/\">《网络爬虫：从入门到实践》一书勘误</a></h1>\n",
      "                    <div class=\"post-meta\">\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-calendar\"></i> 07月11日</span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-folder-open-o\"></i> <a href=\"http://www.santostang.com/category/python-%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"category tag\">Python 网络爬虫</a></span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-commenting-o\"></i> <a href=\"http://www.santostang.com/2018/07/11/%e3%80%8a%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%ef%bc%9a%e4%bb%8e%e5%85%a5%e9%97%a8%e5%88%b0%e5%ae%9e%e8%b7%b5%e3%80%8b%e4%b8%80%e4%b9%a6%e5%8b%98%e8%af%af/#respond\">没有评论</a></span>\n",
      "                        <span class=\"meta-span hidden-xs\"><i class=\"fa fa-tags\" aria-hidden=\"true\"></i> </span>\n",
      "                    </div>\n",
      "                </header>\n",
      "                <div class=\"post-content clearfix\">\n",
      "                    <p>本书由于是第一版，因此还存在一些差错，希望各位读者谅解。\n",
      "\n",
      "另外，感谢各位读者的指正，现将本书的错误之处一并放在此处，方便其他读者更好阅读和使用此书。也欢迎大家在知乎私信或者留言给我，我会持续更新此...</p>\n",
      "                </div>\n",
      "            </article>\n",
      "                                                <article class=\"article-list-1 clearfix\">\n",
      "                <header class=\"clearfix\">\n",
      "                    <h1 class=\"post-title\"><a href=\"http://www.santostang.com/2018/07/04/hello-world/\">Hello world!</a></h1>\n",
      "                    <div class=\"post-meta\">\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-calendar\"></i> 07月04日</span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-folder-open-o\"></i> <a href=\"http://www.santostang.com/category/python-%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" rel=\"category tag\">Python 网络爬虫</a></span>\n",
      "                        <span class=\"meta-span\"><i class=\"fa fa-commenting-o\"></i> <a href=\"http://www.santostang.com/2018/07/04/hello-world/#comments\">1条评论</a></span>\n",
      "                        <span class=\"meta-span hidden-xs\"><i class=\"fa fa-tags\" aria-hidden=\"true\"></i> </span>\n",
      "                    </div>\n",
      "                </header>\n",
      "                <div class=\"post-content clearfix\">\n",
      "                    <p>Welcome to WordPress. This is your first post. Edit or delete it, then start writing!\n",
      "\n",
      "各位读者，由于网易云跟帖在本书出版后已经停止服务，书中的第四章已经无法使用。所以我将本书的评论系统换成了来必力...</p>\n",
      "                </div>\n",
      "            </article>\n",
      "                                    <nav style=\"float:right\">\n",
      "                            </nav>\n",
      "        </div>\n",
      "        <div class=\"col-md-4 hidden-xs hidden-sm\">\n",
      "            <aside class=\"widget clearfix\">\n",
      "    <form id=\"searchform\" action=\"http://www.santostang.com\">\n",
      "        <div class=\"input-group\">\n",
      "            <input type=\"search\" class=\"form-control\" placeholder=\"搜索…\" value=\"\" name=\"s\">\n",
      "            <span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"submit\"><i class=\"fa fa-search\" aria-hidden=\"true\"></i></button></span>\n",
      "        </div>\n",
      "    </form>\n",
      "</aside>\n",
      "<aside class=\"widget clearfix\">\n",
      "    <h4 class=\"widget-title\">文章分类</h4>\n",
      "    <ul class=\"widget-cat\">\n",
      "        \t<li class=\"cat-item cat-item-1\"><a href=\"http://www.santostang.com/category/uncategorized/\" >未分类</a> (1)\n",
      "</li>\n",
      "\t<li class=\"cat-item cat-item-3\"><a href=\"http://www.santostang.com/category/python-%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" >Python 网络爬虫</a> (4)\n",
      "</li>\n",
      "    </ul>\n",
      "</aside>\n",
      "<aside class=\"widget clearfix\">\n",
      "    <h4 class=\"widget-title\">热门文章</h4>\n",
      "    <ul class=\"widget-hot\">\n",
      "        \n",
      "<li><a href=\"http://www.santostang.com/2018/07/04/hello-world/\" target=\"_blank\" >Hello world!</a></li>\n",
      "<li><a href=\"http://www.santostang.com/2018/07/11/%e3%80%8a%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%ef%bc%9a%e4%bb%8e%e5%85%a5%e9%97%a8%e5%88%b0%e5%ae%9e%e8%b7%b5%e3%80%8b%e4%b8%80%e4%b9%a6%e5%8b%98%e8%af%af/\" target=\"_blank\" >《网络爬虫：从入门到实践》一书勘误</a></li>\n",
      "<li><a href=\"http://www.santostang.com/2018/07/14/%e7%ac%ac%e5%9b%9b%e7%ab%a0%ef%bc%9a%e5%8a%a8%e6%80%81%e7%bd%91%e9%a1%b5%e6%8a%93%e5%8f%96-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80-selenium/\" target=\"_blank\" >第四章- 动态网页抓取 (解析真实地址 + selenium)</a></li>\n",
      "<li><a href=\"http://www.santostang.com/2018/07/14/4-2-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80%e6%8a%93%e5%8f%96/\" target=\"_blank\" >4.2 解析真实地址抓取</a></li>\n",
      "<li><a href=\"http://www.santostang.com/2018/07/15/4-3-%e9%80%9a%e8%bf%87selenium-%e6%a8%a1%e6%8b%9f%e6%b5%8f%e8%a7%88%e5%99%a8%e6%8a%93%e5%8f%96/\" target=\"_blank\" >4.3 通过selenium 模拟浏览器抓取</a></li>    </ul>\n",
      "</aside>\n",
      "<aside class=\"widget clearfix\">\n",
      "    <h4 class=\"widget-title\">随机推荐</h4>\n",
      "    <ul class=\"widget-hot\">\n",
      "            <li><a href=\"http://www.santostang.com/2018/07/14/%e7%ac%ac%e5%9b%9b%e7%ab%a0%ef%bc%9a%e5%8a%a8%e6%80%81%e7%bd%91%e9%a1%b5%e6%8a%93%e5%8f%96-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80-selenium/\" title=\"第四章- 动态网页抓取 (解析真实地址 + selenium)\">第四章- 动态网页抓取 (解析真实地址 + selenium)</a></li>\n",
      "            <li><a href=\"http://www.santostang.com/2018/07/04/hello-world/\" title=\"Hello world!\">Hello world!</a></li>\n",
      "            <li><a href=\"http://www.santostang.com/2018/07/15/4-3-%e9%80%9a%e8%bf%87selenium-%e6%a8%a1%e6%8b%9f%e6%b5%8f%e8%a7%88%e5%99%a8%e6%8a%93%e5%8f%96/\" title=\"4.3 通过selenium 模拟浏览器抓取\">4.3 通过selenium 模拟浏览器抓取</a></li>\n",
      "            <li><a href=\"http://www.santostang.com/2018/07/11/%e3%80%8a%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%ef%bc%9a%e4%bb%8e%e5%85%a5%e9%97%a8%e5%88%b0%e5%ae%9e%e8%b7%b5%e3%80%8b%e4%b8%80%e4%b9%a6%e5%8b%98%e8%af%af/\" title=\"《网络爬虫：从入门到实践》一书勘误\">《网络爬虫：从入门到实践》一书勘误</a></li>\n",
      "            <li><a href=\"http://www.santostang.com/2018/07/14/4-2-%e8%a7%a3%e6%9e%90%e7%9c%9f%e5%ae%9e%e5%9c%b0%e5%9d%80%e6%8a%93%e5%8f%96/\" title=\"4.2 解析真实地址抓取\">4.2 解析真实地址抓取</a></li>\n",
      "        </ul>\n",
      "</aside>\n",
      "<aside class=\"widget clearfix\">\n",
      "    <h4 class=\"widget-title\">标签云</h4>\n",
      "    <div class=\"widget-tags\">\n",
      "        <a href=\"http://www.santostang.com/tag/ajax/\" class=\"tag-cloud-link tag-link-7 tag-link-position-1\" style=\"color:#80349f;font-size: 22pt;\" aria-label=\"ajax (2个项目);\">ajax</a>\n",
      "<a href=\"http://www.santostang.com/tag/javascript/\" class=\"tag-cloud-link tag-link-8 tag-link-position-2\" style=\"color:#359914;font-size: 8pt;\" aria-label=\"javascript (1个项目);\">javascript</a>\n",
      "<a href=\"http://www.santostang.com/tag/python/\" class=\"tag-cloud-link tag-link-5 tag-link-position-3\" style=\"color:#e4b80c;font-size: 22pt;\" aria-label=\"python (2个项目);\">python</a>\n",
      "<a href=\"http://www.santostang.com/tag/sselenium/\" class=\"tag-cloud-link tag-link-4 tag-link-position-4\" style=\"color:#cb1705;font-size: 8pt;\" aria-label=\"sselenium (1个项目);\">sselenium</a>\n",
      "<a href=\"http://www.santostang.com/tag/%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab/\" class=\"tag-cloud-link tag-link-6 tag-link-position-5\" style=\"color:#e32c6f;font-size: 22pt;\" aria-label=\"网络爬虫 (2个项目);\">网络爬虫</a>\n",
      "<a href=\"http://www.santostang.com/tag/%e7%bd%91%e9%a1%b5%e7%88%ac%e8%99%ab/\" class=\"tag-cloud-link tag-link-9 tag-link-position-6\" style=\"color:#521dda;font-size: 8pt;\" aria-label=\"网页爬虫 (1个项目);\">网页爬虫</a>\n",
      "<a href=\"http://www.santostang.com/tag/%e8%a7%a3%e6%9e%90%e5%9c%b0%e5%9d%80/\" class=\"tag-cloud-link tag-link-10 tag-link-position-7\" style=\"color:#4563de;font-size: 8pt;\" aria-label=\"解析地址 (1个项目);\">解析地址</a>    </div>\n",
      "</aside>\n",
      "<aside class=\"widget clearfix\">\n",
      "    <h4 class=\"widget-title\">友情链接</h4>\n",
      "    <ul class=\"widget-links\">\n",
      "            </ul>\n",
      "</aside>\n",
      "        </div>\n",
      "    </div>\n",
      "</div>\n",
      "\n",
      "<div class=\"footer_search visible-xs visible-sm\">\n",
      "    <form id=\"searchform\" action=\"http://www.santostang.com\">\n",
      "        <div class=\"input-group\">\n",
      "            <input type=\"search\" class=\"form-control\" placeholder=\"搜索…\" value=\"\" name=\"s\">\n",
      "            <span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"submit\"><i class=\"fa fa-search\" aria-hidden=\"true\"></i></button></span>\n",
      "        </div>\n",
      "    </form>\n",
      "</div>\n",
      "<footer id=\"footer\">\n",
      "    <div class=\"copyright\">\n",
      "        <p><i class=\"fa fa-copyright\" aria-hidden=\"true\"></i> 2018 <b>唐松 Santos</b></p>\n",
      "        <p>Powered by <b>WordPress</b>. Theme by <a href=\"https://tangjie.me/jiestyle-two\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"WordPress 主题模板\" target=\"_blank\"><b>JieStyle Two</b></a> | </p>\n",
      "    </div>\n",
      "    <div style=\"display:none;\">代码在页面底部，统计标识不会显示，但不影响统计效果</div>\n",
      "</footer>\n",
      "<script type=\"text/javascript\" src=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/js/skel.min.js\"></script>\n",
      "<script type=\"text/javascript\" src=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/js/util.min.js\"></script>\n",
      "<script type=\"text/javascript\" src=\"http://www.santostang.com/wp-content/themes/JieStyle-Two-master/js/nav.js\"></script>\n",
      "<script>\n",
      "$(function() {\n",
      "    $('[data-toggle=\"tooltip\"]').tooltip()\n",
      "});\n",
      "</script>\n",
      "<script>\n",
      "(function(){\n",
      "    var bp = document.createElement('script');\n",
      "    var curProtocol = window.location.protocol.split(':')[0];\n",
      "    if (curProtocol === 'https') {\n",
      "        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';\n",
      "    }\n",
      "    else {\n",
      "        bp.src = 'http://push.zhanzhang.baidu.com/push.js';\n",
      "    }\n",
      "    var s = document.getElementsByTagName(\"script\")[0];\n",
      "    s.parentNode.insertBefore(bp, s);\n",
      "})();\n",
      "</script>\n",
      "</body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/python\n",
    "# coding: utf-8\n",
    "\n",
    "import requests #引入包requests\n",
    "link = \"http://www.santostang.com/\"#定义link为目标网页地址\n",
    "# 定义请求头的浏览器代理，伪装成浏览器\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "\n",
    "r = requests.get(link, headers= headers) #请求网页\n",
    "print (r.text)  #r.text是获取的网页内容代码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.2 第二步：提取需要的数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-05T23:33:45.473263Z",
     "start_time": "2018-01-05T23:33:43.470785Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.3 通过selenium 模拟浏览器抓取\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/python\n",
    "# coding: utf-8\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup     #从bs4这个库中导入BeautifulSoup\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")  #使用BeautifulSoup解析这段代码\n",
    "#找到第一篇文章标题，定位到class是\"post-title\"的h1元素，提取a，提取里面的字符串，strip()去除左右空格\n",
    "title = soup.find(\"h1\", class_=\"post-title\").a.text.strip()\n",
    "print (title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.3 第三步：储存数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.3 通过selenium 模拟浏览器抓取\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup   #从bs4这个库中导入BeautifulSoup\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")   #使用BeautifulSoup解析这段代码\n",
    "title = soup.find(\"h1\", class_=\"post-title\").a.text.strip()\n",
    "print (title)\n",
    "\n",
    "# 打开一个空白的txt，然后使用f.write写入刚刚的字符串title\n",
    "with open('title_test.txt', \"a+\") as f:\n",
    "    f.write(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "192px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
